


<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Automatic differentiation package - torch.autograd &mdash; PyTorch master documentation</title>
  

  
  
  
  
    <link rel="canonical" href="https://pytorch.org/docs/stable/autograd.html"/>
  

  

  
  
    

  

  <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="_static/css/jit.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.11.1/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="_static/katex-math.css" type="text/css" />
    <link rel="index" title="Index" href="genindex.html" />
    <link rel="search" title="Search" href="search.html" />
    <link rel="next" title="torch.cuda" href="cuda.html" />
    <link rel="prev" title="Tensor Views" href="tensor_view.html" /> 

  
  <script src="_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <div class="ecosystem-dropdown">
              <a id="dropdownMenuButton" data-toggle="ecosystem-dropdown">
                Ecosystem
              </a>
              <div class="ecosystem-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/hub"">
                  <span class=dropdown-title>Models (Beta)</span>
                  <p>Discover, publish, and reuse pre-trained models</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/ecosystem">
                  <span class=dropdown-title>Tools & Libraries</span>
                  <p>Explore the ecosystem of tools and libraries</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <div class="resources-dropdown">
              <a id="resourcesDropdownButton" data-toggle="resources-dropdown">
                Resources
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/resources"">
                  <span class=dropdown-title>Developer Resources</span>
                  <p>Find resources and get questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/features">
                  <span class=dropdown-title>About</span>
                  <p>Learn about PyTorch’s features and capabilities</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>

  </div>
</div>


<body class="pytorch-body">

   

    

    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            

            
              
              
                <div class="version">
                  master (1.5.0 )
                </div>
              
            

            


  


<div role="search">
  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

            
          </div>

          
<div>
  <a style="color:#F05732" href="https://pytorch.org/docs/stable/autograd.html">
    You are viewing unstable developer preview docs.
    Click here to view docs for latest stable release.
  </a>
</div>

            
            
              
            
            
              <p class="caption"><span class="caption-text">Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="notes/amp_examples.html">Automatic Mixed Precision examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/autograd.html">Autograd mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/broadcasting.html">Broadcasting semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/cpu_threading_torchscript_inference.html">CPU threading and TorchScript inference</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/cuda.html">CUDA semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/ddp.html">Distributed Data Parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/extending.html">Extending PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/large_scale_deployments.html">Features for large-scale deployments</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/multiprocessing.html">Multiprocessing best practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/randomness.html">Reproducibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/serialization.html">Serialization semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/windows.html">Windows FAQ</a></li>
</ul>
<p class="caption"><span class="caption-text">Language Bindings</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/cppdocs/">C++ API</a></li>
<li class="toctree-l1"><a class="reference internal" href="packages.html">Javadoc</a></li>
</ul>
<p class="caption"><span class="caption-text">Python API</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="torch.html">torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="nn.html">torch.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="nn.functional.html">torch.nn.functional</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensors.html">torch.Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensor_attributes.html">Tensor Attributes</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensor_view.html">Tensor Views</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">torch.autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="cuda.html">torch.cuda</a></li>
<li class="toctree-l1"><a class="reference internal" href="amp.html">torch.cuda.amp</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributed.html">torch.distributed</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributions.html">torch.distributions</a></li>
<li class="toctree-l1"><a class="reference internal" href="hub.html">torch.hub</a></li>
<li class="toctree-l1"><a class="reference internal" href="jit.html">torch.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="nn.init.html">torch.nn.init</a></li>
<li class="toctree-l1"><a class="reference internal" href="onnx.html">torch.onnx</a></li>
<li class="toctree-l1"><a class="reference internal" href="optim.html">torch.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="quantization.html">Quantization</a></li>
<li class="toctree-l1"><a class="reference internal" href="rpc/index.html">Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="random.html">torch.random</a></li>
<li class="toctree-l1"><a class="reference internal" href="sparse.html">torch.sparse</a></li>
<li class="toctree-l1"><a class="reference internal" href="storage.html">torch.Storage</a></li>
<li class="toctree-l1"><a class="reference internal" href="bottleneck.html">torch.utils.bottleneck</a></li>
<li class="toctree-l1"><a class="reference internal" href="checkpoint.html">torch.utils.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="cpp_extension.html">torch.utils.cpp_extension</a></li>
<li class="toctree-l1"><a class="reference internal" href="data.html">torch.utils.data</a></li>
<li class="toctree-l1"><a class="reference internal" href="dlpack.html">torch.utils.dlpack</a></li>
<li class="toctree-l1"><a class="reference internal" href="model_zoo.html">torch.utils.model_zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensorboard.html">torch.utils.tensorboard</a></li>
<li class="toctree-l1"><a class="reference internal" href="type_info.html">Type Info</a></li>
<li class="toctree-l1"><a class="reference internal" href="named_tensor.html">Named Tensors</a></li>
<li class="toctree-l1"><a class="reference internal" href="name_inference.html">Named Tensors operator coverage</a></li>
<li class="toctree-l1"><a class="reference internal" href="__config__.html">torch.__config__</a></li>
</ul>
<p class="caption"><span class="caption-text">Libraries</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/audio">torchaudio</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/text">torchtext</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/elastic/">TorchElastic</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/serve">TorchServe</a></li>
<li class="toctree-l1"><a class="reference external" href="http://pytorch.org/xla/">PyTorch on XLA Devices</a></li>
</ul>
<p class="caption"><span class="caption-text">Community</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="community/contribution_guide.html">PyTorch Contribution Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="community/governance.html">PyTorch Governance</a></li>
<li class="toctree-l1"><a class="reference internal" href="community/persons_of_interest.html">PyTorch Governance | Persons of Interest</a></li>
</ul>

            
          

        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="index.html">
          
            Docs
          
        </a> &gt;
      </li>

        
      <li>Automatic differentiation package - torch.autograd</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
            
            <a href="_sources/autograd.rst.txt" rel="nofollow"><img src="_static/images/view-page-source-icon.svg"></a>
          
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <div class="section" id="module-torch.autograd">
<span id="automatic-differentiation-package-torch-autograd"></span><h1>Automatic differentiation package - torch.autograd<a class="headerlink" href="#module-torch.autograd" title="Permalink to this headline">¶</a></h1>
<p><code class="docutils literal notranslate"><span class="pre">torch.autograd</span></code> provides classes and functions implementing automatic
differentiation of arbitrary scalar valued functions. It requires minimal
changes to the existing code - you only need to declare <code class="xref py py-class docutils literal notranslate"><span class="pre">Tensor</span></code> s
for which gradients should be computed with the <code class="docutils literal notranslate"><span class="pre">requires_grad=True</span></code> keyword.</p>
<dl class="function">
<dt id="torch.autograd.backward">
<code class="sig-prename descclassname">torch.autograd.</code><code class="sig-name descname">backward</code><span class="sig-paren">(</span><em class="sig-param">tensors</em>, <em class="sig-param">grad_tensors=None</em>, <em class="sig-param">retain_graph=None</em>, <em class="sig-param">create_graph=False</em>, <em class="sig-param">grad_variables=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd.html#backward"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.backward" title="Permalink to this definition">¶</a></dt>
<dd><p>Computes the sum of gradients of given tensors w.r.t. graph leaves.</p>
<p>The graph is differentiated using the chain rule. If any of <code class="docutils literal notranslate"><span class="pre">tensors</span></code>
are non-scalar (i.e. their data has more than one element) and require
gradient, then the Jacobian-vector product would be computed, in this
case the function additionally requires specifying <code class="docutils literal notranslate"><span class="pre">grad_tensors</span></code>.
It should be a sequence of matching length, that contains the “vector”
in the Jacobian-vector product, usually the gradient of the differentiated
function w.r.t. corresponding tensors (<code class="docutils literal notranslate"><span class="pre">None</span></code> is an acceptable value for
all tensors that don’t need gradient tensors).</p>
<p>This function accumulates gradients in the leaves - you might need to zero
them before calling it.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensors</strong> (<em>sequence of Tensor</em>) – Tensors of which the derivative will be
computed.</p></li>
<li><p><strong>grad_tensors</strong> (<em>sequence of</em><em> (</em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.8)"><em>None</em></a><em>)</em>) – The “vector” in the Jacobian-vector
product, usually gradients w.r.t. each element of corresponding tensors.
None values can be specified for scalar Tensors or ones that don’t require
grad. If a None value would be acceptable for all grad_tensors, then this
argument is optional.</p></li>
<li><p><strong>retain_graph</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">False</span></code>, the graph used to compute the grad
will be freed. Note that in nearly all cases setting this option to <code class="docutils literal notranslate"><span class="pre">True</span></code>
is not needed and often can be worked around in a much more efficient
way. Defaults to the value of <code class="docutils literal notranslate"><span class="pre">create_graph</span></code>.</p></li>
<li><p><strong>create_graph</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, graph of the derivative will
be constructed, allowing to compute higher order derivative products.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="function">
<dt id="torch.autograd.grad">
<code class="sig-prename descclassname">torch.autograd.</code><code class="sig-name descname">grad</code><span class="sig-paren">(</span><em class="sig-param">outputs</em>, <em class="sig-param">inputs</em>, <em class="sig-param">grad_outputs=None</em>, <em class="sig-param">retain_graph=None</em>, <em class="sig-param">create_graph=False</em>, <em class="sig-param">only_inputs=True</em>, <em class="sig-param">allow_unused=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd.html#grad"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.grad" title="Permalink to this definition">¶</a></dt>
<dd><p>Computes and returns the sum of gradients of outputs w.r.t. the inputs.</p>
<p><code class="docutils literal notranslate"><span class="pre">grad_outputs</span></code> should be a sequence of length matching <code class="docutils literal notranslate"><span class="pre">output</span></code>
containing the “vector” in Jacobian-vector product, usually the pre-computed
gradients w.r.t. each of the outputs. If an output doesn’t require_grad,
then the gradient can be <code class="docutils literal notranslate"><span class="pre">None</span></code>).</p>
<p>If <code class="docutils literal notranslate"><span class="pre">only_inputs</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, the function will only return a list of gradients
w.r.t the specified inputs. If it’s <code class="docutils literal notranslate"><span class="pre">False</span></code>, then gradient w.r.t. all remaining
leaves will still be computed, and will be accumulated into their <code class="docutils literal notranslate"><span class="pre">.grad</span></code>
attribute.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>outputs</strong> (<em>sequence of Tensor</em>) – outputs of the differentiated function.</p></li>
<li><p><strong>inputs</strong> (<em>sequence of Tensor</em>) – Inputs w.r.t. which the gradient will be
returned (and not accumulated into <code class="docutils literal notranslate"><span class="pre">.grad</span></code>).</p></li>
<li><p><strong>grad_outputs</strong> (<em>sequence of Tensor</em>) – The “vector” in the Jacobian-vector product.
Usually gradients w.r.t. each output. None values can be specified for scalar
Tensors or ones that don’t require grad. If a None value would be acceptable
for all grad_tensors, then this argument is optional. Default: None.</p></li>
<li><p><strong>retain_graph</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">False</span></code>, the graph used to compute the grad
will be freed. Note that in nearly all cases setting this option to <code class="docutils literal notranslate"><span class="pre">True</span></code>
is not needed and often can be worked around in a much more efficient
way. Defaults to the value of <code class="docutils literal notranslate"><span class="pre">create_graph</span></code>.</p></li>
<li><p><strong>create_graph</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, graph of the derivative will
be constructed, allowing to compute higher order derivative products.
Default: <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>allow_unused</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">False</span></code>, specifying inputs that were not
used when computing outputs (and therefore their grad is always zero)
is an error. Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<div class="section" id="functional-higher-level-api">
<span id="functional-api"></span><h2>Functional higher level API<a class="headerlink" href="#functional-higher-level-api" title="Permalink to this headline">¶</a></h2>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This API is experimental.</p>
</div>
<p>This section contains the higher level API for the autograd that builds on the basic API above
and allows you to compute jacobians, hessians, etc.</p>
<p>This API works with user-provided functions that take only Tensors as input and return
only Tensors.
If your function takes other arguments that are not Tensors or Tensors for which you don’t require gradients,
you can use a lambda to capture them.
For example, for a function <code class="docutils literal notranslate"><span class="pre">f</span></code> that takes three inputs, a Tensor for which we want the jacobian, another
tensor that should be considered constant and a boolean flag as <code class="docutils literal notranslate"><span class="pre">f(input,</span> <span class="pre">constant,</span> <span class="pre">flag=flag)</span></code>
you can use it as <code class="docutils literal notranslate"><span class="pre">functional.jacobian(lambda</span> <span class="pre">x:</span> <span class="pre">f(x,</span> <span class="pre">constant,</span> <span class="pre">flag=flag),</span> <span class="pre">input)</span></code>.</p>
<dl class="function">
<dt id="torch.autograd.functional.jacobian">
<code class="sig-prename descclassname">torch.autograd.functional.</code><code class="sig-name descname">jacobian</code><span class="sig-paren">(</span><em class="sig-param">func</em>, <em class="sig-param">inputs</em>, <em class="sig-param">create_graph=False</em>, <em class="sig-param">strict=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/functional.html#jacobian"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.functional.jacobian" title="Permalink to this definition">¶</a></dt>
<dd><p>Function that computes the Jacobian of a given function.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>func</strong> (<em>function</em>) – a Python function that takes Tensor inputs and returns
a tuple of Tensors or a Tensor.</p></li>
<li><p><strong>inputs</strong> (<em>tuple of Tensors</em><em> or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – inputs to the function <code class="docutils literal notranslate"><span class="pre">func</span></code>.</p></li>
<li><p><strong>create_graph</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, the Jacobian will be computed in
a differentiable manner. Note that when <code class="docutils literal notranslate"><span class="pre">strict</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, the result can not
require gradients or be disconnected from the inputs.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>strict</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, an error will be raised when we detect that there exists an input
such that all the outputs are independent of it. If <code class="docutils literal notranslate"><span class="pre">False</span></code>, we return a Tensor of zeros as the
jacobian for said inputs, which is the expected mathematical value.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p><dl class="simple">
<dt>Jacobian (Tensor or nested tuple of Tensors) if there are a single input</dt><dd><p>and output, this will be a single Tensor containing the Jacobian for the
linearized inputs and output. If one of the two is a tuple, then the Jacobian
will be a tuple of Tensors. If both of them are tuples, then the Jacobian will
be a tuple of tuple of Tensors where <code class="docutils literal notranslate"><span class="pre">Jacobian[i][j]</span></code> will contain the Jacobian
of the <a href="#id1"><span class="problematic" id="id2">``</span></a>i``th output and <a href="#id3"><span class="problematic" id="id4">``</span></a>j``th input and will have as size the concatenation of the
sizes of the corresponding output and the corresponding input.</p>
</dd>
</dl>
</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">exp_reducer</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
<span class="gp">... </span>  <span class="k">return</span> <span class="n">x</span><span class="o">.</span><span class="n">exp</span><span class="p">()</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">jacobian</span><span class="p">(</span><span class="n">exp_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">)</span>
<span class="go">tensor([[[1.4917, 2.4352],</span>
<span class="go">         [0.0000, 0.0000]],</span>

<span class="go">        [[0.0000, 0.0000],</span>
<span class="go">         [2.4369, 2.3799]]])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">jacobian</span><span class="p">(</span><span class="n">exp_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">create_graph</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="go">tensor([[[1.4917, 2.4352],</span>
<span class="go">         [0.0000, 0.0000]],</span>

<span class="go">        [[0.0000, 0.0000],</span>
<span class="go">         [2.4369, 2.3799]]], grad_fn=&lt;ViewBackward&gt;)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">exp_adder</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">):</span>
<span class="gp">... </span>  <span class="k">return</span> <span class="mi">2</span> <span class="o">*</span> <span class="n">x</span><span class="o">.</span><span class="n">exp</span><span class="p">()</span> <span class="o">+</span> <span class="mi">3</span> <span class="o">*</span> <span class="n">y</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">jacobian</span><span class="p">(</span><span class="n">exp_adder</span><span class="p">,</span> <span class="n">inputs</span><span class="p">)</span>
<span class="go">(tensor([[2.8052, 0.0000],</span>
<span class="go">        [0.0000, 3.3963]]),</span>
<span class="go"> tensor([[3., 0.],</span>
<span class="go">         [0., 3.]]))</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.autograd.functional.hessian">
<code class="sig-prename descclassname">torch.autograd.functional.</code><code class="sig-name descname">hessian</code><span class="sig-paren">(</span><em class="sig-param">func</em>, <em class="sig-param">inputs</em>, <em class="sig-param">create_graph=False</em>, <em class="sig-param">strict=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/functional.html#hessian"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.functional.hessian" title="Permalink to this definition">¶</a></dt>
<dd><p>Function that computes the Hessian of a given scalar function.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>func</strong> (<em>function</em>) – a Python function that takes Tensor inputs and returns
a Tensor with a single element.</p></li>
<li><p><strong>inputs</strong> (<em>tuple of Tensors</em><em> or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – inputs to the function <code class="docutils literal notranslate"><span class="pre">func</span></code>.</p></li>
<li><p><strong>create_graph</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, the Hessian will be computed in
a differentiable manner. Note that when <code class="docutils literal notranslate"><span class="pre">strict</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, the result can not
require gradients or be disconnected from the inputs.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>strict</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, an error will be raised when we detect that there exists an input
such that all the outputs are independent of it. If <code class="docutils literal notranslate"><span class="pre">False</span></code>, we return a Tensor of zeros as the
hessian for said inputs, which is the expected mathematical value.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p><dl class="simple">
<dt>Hessian (Tensor or a tuple of tuple of Tensors) if there are a single input,</dt><dd><p>this will be a single Tensor containing the Hessian for the input.
If it is a tuple, then the Hessian will be a tuple of tuples where
<code class="docutils literal notranslate"><span class="pre">Hessian[i][j]</span></code> will contain the Hessian of the <a href="#id5"><span class="problematic" id="id6">``</span></a>i``th input
and <a href="#id7"><span class="problematic" id="id8">``</span></a>j``th input with size the sum of the size of the <a href="#id9"><span class="problematic" id="id10">``</span></a>i``th input plus
the size of the <a href="#id11"><span class="problematic" id="id12">``</span></a>j``th input.</p>
</dd>
</dl>
</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">pow_reducer</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
<span class="gp">... </span>  <span class="k">return</span> <span class="n">x</span><span class="o">.</span><span class="n">pow</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">hessian</span><span class="p">(</span><span class="n">pow_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">)</span>
<span class="go">tensor([[[[5.2265, 0.0000],</span>
<span class="go">          [0.0000, 0.0000]],</span>

<span class="go">         [[0.0000, 4.8221],</span>
<span class="go">          [0.0000, 0.0000]]],</span>


<span class="go">        [[[0.0000, 0.0000],</span>
<span class="go">          [1.9456, 0.0000]],</span>

<span class="go">         [[0.0000, 0.0000],</span>
<span class="go">          [0.0000, 3.2550]]]])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">hessian</span><span class="p">(</span><span class="n">pow_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">create_graph</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="go">tensor([[[[5.2265, 0.0000],</span>
<span class="go">          [0.0000, 0.0000]],</span>

<span class="go">         [[0.0000, 4.8221],</span>
<span class="go">          [0.0000, 0.0000]]],</span>


<span class="go">        [[[0.0000, 0.0000],</span>
<span class="go">          [1.9456, 0.0000]],</span>

<span class="go">         [[0.0000, 0.0000],</span>
<span class="go">          [0.0000, 3.2550]]]], grad_fn=&lt;ViewBackward&gt;)</span>


<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">pow_adder_reducer</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">):</span>
<span class="gp">... </span>  <span class="k">return</span> <span class="p">(</span><span class="mi">2</span> <span class="o">*</span> <span class="n">x</span><span class="o">.</span><span class="n">pow</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mi">3</span> <span class="o">*</span> <span class="n">y</span><span class="o">.</span><span class="n">pow</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">hessian</span><span class="p">(</span><span class="n">pow_adder_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">)</span>
<span class="go">((tensor([[4., 0.],</span>
<span class="go">          [0., 4.]]),</span>
<span class="go">  tensor([[0., 0.],</span>
<span class="go">          [0., 0.]])),</span>
<span class="go"> (tensor([[0., 0.],</span>
<span class="go">          [0., 0.]]),</span>
<span class="go">  tensor([[6., 0.],</span>
<span class="go">          [0., 6.]])))</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.autograd.functional.vjp">
<code class="sig-prename descclassname">torch.autograd.functional.</code><code class="sig-name descname">vjp</code><span class="sig-paren">(</span><em class="sig-param">func</em>, <em class="sig-param">inputs</em>, <em class="sig-param">v=None</em>, <em class="sig-param">create_graph=False</em>, <em class="sig-param">strict=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/functional.html#vjp"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.functional.vjp" title="Permalink to this definition">¶</a></dt>
<dd><p>Function that computes the dot product between a vector <code class="docutils literal notranslate"><span class="pre">v</span></code> and the Jacobian of
the given function at the point given by the inputs.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>func</strong> (<em>function</em>) – a Python function that takes Tensor inputs and returns
a tuple of Tensors or a Tensor.</p></li>
<li><p><strong>inputs</strong> (<em>tuple of Tensors</em><em> or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – inputs to the function <code class="docutils literal notranslate"><span class="pre">func</span></code>.</p></li>
<li><p><strong>v</strong> (<em>tuple of Tensors</em><em> or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – The vector for which the vector Jacobian product is computed.
Must be the same size as the output of <code class="docutils literal notranslate"><span class="pre">func</span></code>. This argument is optional when
<code class="docutils literal notranslate"><span class="pre">func</span></code>’s output contains a single element and (if it is not provided) will be set as a Tensor
containing a single <code class="docutils literal notranslate"><span class="pre">1</span></code>.</p></li>
<li><p><strong>create_graph</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, both the output and result will be
computed in a differentiable way. Note that when <code class="docutils literal notranslate"><span class="pre">strict</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, the result can not
require gradients or be disconnected from the inputs.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>strict</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, an error will be raised when we detect that there exists an input
such that all the outputs are independent of it. If <code class="docutils literal notranslate"><span class="pre">False</span></code>, we return a Tensor of zeros as the
vjp for said inputs, which is the expected mathematical value.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p><p>output of <code class="docutils literal notranslate"><span class="pre">func(inputs)</span></code>
vjp (tuple of Tensors or Tensor): result of the dot product with the same shape</p>
<blockquote>
<div><p>as the inputs.</p>
</div></blockquote>
</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p>func_output (tuple of Tensors or <a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor">Tensor</a>)</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">exp_reducer</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
<span class="gp">... </span>  <span class="k">return</span> <span class="n">x</span><span class="o">.</span><span class="n">exp</span><span class="p">()</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">v</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">4</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">vjp</span><span class="p">(</span><span class="n">exp_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
<span class="go">(tensor([5.7817, 7.2458, 5.7830, 6.7782]),</span>
<span class="go"> tensor([[1.4458, 1.3962, 1.3042, 1.6354],</span>
<span class="go">        [2.1288, 1.0652, 1.5483, 2.5035],</span>
<span class="go">        [2.2046, 1.1292, 1.1432, 1.3059],</span>
<span class="go">        [1.3225, 1.6652, 1.7753, 2.0152]]))</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">vjp</span><span class="p">(</span><span class="n">exp_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">v</span><span class="p">,</span> <span class="n">create_graph</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="go">(tensor([5.7817, 7.2458, 5.7830, 6.7782], grad_fn=&lt;SumBackward1&gt;),</span>
<span class="go"> tensor([[1.4458, 1.3962, 1.3042, 1.6354],</span>
<span class="go">        [2.1288, 1.0652, 1.5483, 2.5035],</span>
<span class="go">        [2.2046, 1.1292, 1.1432, 1.3059],</span>
<span class="go">        [1.3225, 1.6652, 1.7753, 2.0152]], grad_fn=&lt;MulBackward0&gt;))</span>

<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">adder</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">):</span>
<span class="gp">... </span>  <span class="k">return</span> <span class="mi">2</span> <span class="o">*</span> <span class="n">x</span> <span class="o">+</span> <span class="mi">3</span> <span class="o">*</span> <span class="n">y</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">v</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">vjp</span><span class="p">(</span><span class="n">adder</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
<span class="go">(tensor([2.4225, 2.3340]),</span>
<span class="go"> (tensor([2., 2.]), tensor([3., 3.])))</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.autograd.functional.jvp">
<code class="sig-prename descclassname">torch.autograd.functional.</code><code class="sig-name descname">jvp</code><span class="sig-paren">(</span><em class="sig-param">func</em>, <em class="sig-param">inputs</em>, <em class="sig-param">v=None</em>, <em class="sig-param">create_graph=False</em>, <em class="sig-param">strict=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/functional.html#jvp"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.functional.jvp" title="Permalink to this definition">¶</a></dt>
<dd><p>Function that computes the dot product between  the Jacobian of
the given function at the point given by the inputs and a vector <code class="docutils literal notranslate"><span class="pre">v</span></code>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>func</strong> (<em>function</em>) – a Python function that takes Tensor inputs and returns
a tuple of Tensors or a Tensor.</p></li>
<li><p><strong>inputs</strong> (<em>tuple of Tensors</em><em> or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – inputs to the function <code class="docutils literal notranslate"><span class="pre">func</span></code>.</p></li>
<li><p><strong>v</strong> (<em>tuple of Tensors</em><em> or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – The vector for which the Jacobian vector product is computed. Must be the
same size as the input of <code class="docutils literal notranslate"><span class="pre">func</span></code>. This argument is optional when
<code class="docutils literal notranslate"><span class="pre">func</span></code>’s input contains a single element and (if it is not provided) will be set as a Tensor
containing a single <code class="docutils literal notranslate"><span class="pre">1</span></code>.</p></li>
<li><p><strong>create_graph</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, both the output and result will be
computed in a differentiable way. Note that when <code class="docutils literal notranslate"><span class="pre">strict</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, the result can not
require gradients or be disconnected from the inputs.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>strict</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, an error will be raised when we detect that there exists an input
such that all the outputs are independent of it. If <code class="docutils literal notranslate"><span class="pre">False</span></code>, we return a Tensor of zeros as the
jvp for said inputs, which is the expected mathematical value.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p><p>output of <code class="docutils literal notranslate"><span class="pre">func(inputs)</span></code>
jvp (tuple of Tensors or Tensor): result of the dot product with the same shape</p>
<blockquote>
<div><p>as the output.</p>
</div></blockquote>
</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p>func_output (tuple of Tensors or <a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor">Tensor</a>)</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">exp_reducer</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
<span class="gp">... </span>  <span class="k">return</span> <span class="n">x</span><span class="o">.</span><span class="n">exp</span><span class="p">()</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">v</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">jvp</span><span class="p">(</span><span class="n">exp_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
<span class="go">(tensor([6.3090, 4.6742, 7.9114, 8.2106]),</span>
<span class="go"> tensor([6.3090, 4.6742, 7.9114, 8.2106]))</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">jvp</span><span class="p">(</span><span class="n">exp_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">v</span><span class="p">,</span> <span class="n">create_graph</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="go">(tensor([6.3090, 4.6742, 7.9114, 8.2106], grad_fn=&lt;SumBackward1&gt;),</span>
<span class="go"> tensor([6.3090, 4.6742, 7.9114, 8.2106], grad_fn=&lt;SqueezeBackward1&gt;))</span>

<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">adder</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">):</span>
<span class="gp">... </span>  <span class="k">return</span> <span class="mi">2</span> <span class="o">*</span> <span class="n">x</span> <span class="o">+</span> <span class="mi">3</span> <span class="o">*</span> <span class="n">y</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">v</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">jvp</span><span class="p">(</span><span class="n">adder</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
<span class="go">(tensor([2.2399, 2.5005]),</span>
<span class="go"> tensor([5., 5.]))</span>
</pre></div>
</div>
<p>Note:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">The</span> <span class="n">jvp</span> <span class="ow">is</span> <span class="n">currently</span> <span class="n">computed</span> <span class="n">by</span> <span class="n">using</span> <span class="n">the</span> <span class="n">backward</span> <span class="n">of</span> <span class="n">the</span> <span class="n">backward</span> <span class="p">(</span><span class="n">sometimes</span> <span class="n">called</span> <span class="n">the</span> <span class="n">double</span>
<span class="n">backwards</span> <span class="n">trick</span><span class="p">)</span> <span class="k">as</span> <span class="n">we</span> <span class="n">don</span><span class="s1">&#39;t have support for forward mode AD in PyTorch at the moment.</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.autograd.functional.vhp">
<code class="sig-prename descclassname">torch.autograd.functional.</code><code class="sig-name descname">vhp</code><span class="sig-paren">(</span><em class="sig-param">func</em>, <em class="sig-param">inputs</em>, <em class="sig-param">v=None</em>, <em class="sig-param">create_graph=False</em>, <em class="sig-param">strict=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/functional.html#vhp"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.functional.vhp" title="Permalink to this definition">¶</a></dt>
<dd><p>Function that computes the dot product between a vector <code class="docutils literal notranslate"><span class="pre">v</span></code> and the
Hessian of a given scalar function at the point given by the inputs.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>func</strong> (<em>function</em>) – a Python function that takes Tensor inputs and returns
a Tensor with a single element.</p></li>
<li><p><strong>inputs</strong> (<em>tuple of Tensors</em><em> or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – inputs to the function <code class="docutils literal notranslate"><span class="pre">func</span></code>.</p></li>
<li><p><strong>v</strong> (<em>tuple of Tensors</em><em> or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – The vector for which the vector Hessian product is computed. Must be the
same size as the input of <code class="docutils literal notranslate"><span class="pre">func</span></code>. This argument is optional when
<code class="docutils literal notranslate"><span class="pre">func</span></code>’s input contains a single element and (if it is not provided) will be set as a Tensor
containing a single <code class="docutils literal notranslate"><span class="pre">1</span></code>.</p></li>
<li><p><strong>create_graph</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, both the output and result will be
computed in a differentiable way. Note that when <code class="docutils literal notranslate"><span class="pre">strict</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, the result can not
require gradients or be disconnected from the inputs.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>strict</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, an error will be raised when we detect that there exists an input
such that all the outputs are independent of it. If <code class="docutils literal notranslate"><span class="pre">False</span></code>, we return a Tensor of zeros as the
vhp for said inputs, which is the expected mathematical value.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p><p>output of <code class="docutils literal notranslate"><span class="pre">func(inputs)</span></code>
vhp (tuple of Tensors or Tensor): result of the dot product with the same shape</p>
<blockquote>
<div><p>as the inputs.</p>
</div></blockquote>
</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p>func_output (tuple of Tensors or <a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor">Tensor</a>)</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span> <span class="o">&gt;&gt;&gt;</span> <span class="k">def</span> <span class="nf">pow_reducer</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
 <span class="o">...</span>   <span class="k">return</span> <span class="n">x</span><span class="o">.</span><span class="n">pow</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
 <span class="o">&gt;&gt;&gt;</span> <span class="n">inputs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
 <span class="o">&gt;&gt;&gt;</span> <span class="n">v</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
 <span class="o">&gt;&gt;&gt;</span> <span class="n">vhp</span><span class="p">(</span><span class="n">pow_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
<span class="p">(</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.5591</span><span class="p">),</span>
 <span class="n">tensor</span><span class="p">([[</span><span class="mf">1.0689</span><span class="p">,</span> <span class="mf">1.2431</span><span class="p">],</span>
         <span class="p">[</span><span class="mf">3.0989</span><span class="p">,</span> <span class="mf">4.4456</span><span class="p">]]))</span>

 <span class="o">&gt;&gt;&gt;</span> <span class="n">vhp</span><span class="p">(</span><span class="n">pow_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">v</span><span class="p">,</span> <span class="n">create_graph</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
 <span class="p">(</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.5591</span><span class="p">,</span> <span class="n">grad_fn</span><span class="o">=&lt;</span><span class="n">SumBackward0</span><span class="o">&gt;</span><span class="p">),</span>
  <span class="n">tensor</span><span class="p">([[</span><span class="mf">1.0689</span><span class="p">,</span> <span class="mf">1.2431</span><span class="p">],</span>
          <span class="p">[</span><span class="mf">3.0989</span><span class="p">,</span> <span class="mf">4.4456</span><span class="p">]],</span> <span class="n">grad_fn</span><span class="o">=&lt;</span><span class="n">MulBackward0</span><span class="o">&gt;</span><span class="p">))</span>


 <span class="o">&gt;&gt;&gt;</span> <span class="k">def</span> <span class="nf">pow_adder_reducer</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">):</span>
 <span class="o">...</span>   <span class="k">return</span> <span class="p">(</span><span class="mi">2</span> <span class="o">*</span> <span class="n">x</span><span class="o">.</span><span class="n">pow</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mi">3</span> <span class="o">*</span> <span class="n">y</span><span class="o">.</span><span class="n">pow</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
 <span class="o">&gt;&gt;&gt;</span> <span class="n">inputs</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span>
 <span class="o">&gt;&gt;&gt;</span> <span class="n">v</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span>
 <span class="o">&gt;&gt;&gt;</span> <span class="n">vhp</span><span class="p">(</span><span class="n">pow_adder_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
 <span class="p">(</span><span class="n">tensor</span><span class="p">(</span><span class="mf">4.8053</span><span class="p">),</span>
  <span class="p">(</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.</span><span class="p">,</span> <span class="mf">0.</span><span class="p">]),</span>
   <span class="n">tensor</span><span class="p">([</span><span class="mf">6.</span><span class="p">,</span> <span class="mf">6.</span><span class="p">])))</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.autograd.functional.hvp">
<code class="sig-prename descclassname">torch.autograd.functional.</code><code class="sig-name descname">hvp</code><span class="sig-paren">(</span><em class="sig-param">func</em>, <em class="sig-param">inputs</em>, <em class="sig-param">v=None</em>, <em class="sig-param">create_graph=False</em>, <em class="sig-param">strict=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/functional.html#hvp"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.functional.hvp" title="Permalink to this definition">¶</a></dt>
<dd><p>Function that computes the dot product between the Hessian of a given scalar
function and a vector <code class="docutils literal notranslate"><span class="pre">v</span></code> at the point given by the inputs.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>func</strong> (<em>function</em>) – a Python function that takes Tensor inputs and returns
a Tensor with a single element.</p></li>
<li><p><strong>inputs</strong> (<em>tuple of Tensors</em><em> or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – inputs to the function <code class="docutils literal notranslate"><span class="pre">func</span></code>.</p></li>
<li><p><strong>v</strong> (<em>tuple of Tensors</em><em> or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – The vector for which the Hessian vector product is computed. Must be the
same size as the input of <code class="docutils literal notranslate"><span class="pre">func</span></code>. This argument is optional when
<code class="docutils literal notranslate"><span class="pre">func</span></code>’s input contains a single element and (if it is not provided) will be set as a Tensor
containing a single <code class="docutils literal notranslate"><span class="pre">1</span></code>.</p></li>
<li><p><strong>create_graph</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, both the output and result will be
computed in a differentiable way. Note that when <code class="docutils literal notranslate"><span class="pre">strict</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, the result can not
require gradients or be disconnected from the inputs.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>strict</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, an error will be raised when we detect that there exists an input
such that all the outputs are independent of it. If <code class="docutils literal notranslate"><span class="pre">False</span></code>, we return a Tensor of zeros as the
hvp for said inputs, which is the expected mathematical value.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p><p>output of <code class="docutils literal notranslate"><span class="pre">func(inputs)</span></code>
hvp (tuple of Tensors or Tensor): result of the dot product with the same shape</p>
<blockquote>
<div><p>as the inputs.</p>
</div></blockquote>
</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p>func_output (tuple of Tensors or <a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor">Tensor</a>)</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">pow_reducer</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
<span class="gp">... </span>  <span class="k">return</span> <span class="n">x</span><span class="o">.</span><span class="n">pow</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">v</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">hvp</span><span class="p">(</span><span class="n">pow_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
<span class="go">(tensor(0.1448),</span>
<span class="go"> tensor([[2.0239, 1.6456],</span>
<span class="go">         [2.4988, 1.4310]]))</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">hvp</span><span class="p">(</span><span class="n">pow_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">v</span><span class="p">,</span> <span class="n">create_graph</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="go">(tensor(0.1448, grad_fn=&lt;SumBackward0&gt;),</span>
<span class="go"> tensor([[2.0239, 1.6456],</span>
<span class="go">         [2.4988, 1.4310]], grad_fn=&lt;MulBackward0&gt;))</span>


<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">pow_adder_reducer</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">):</span>
<span class="gp">... </span>  <span class="k">return</span> <span class="p">(</span><span class="mi">2</span> <span class="o">*</span> <span class="n">x</span><span class="o">.</span><span class="n">pow</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mi">3</span> <span class="o">*</span> <span class="n">y</span><span class="o">.</span><span class="n">pow</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">v</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">hvp</span><span class="p">(</span><span class="n">pow_adder_reducer</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
<span class="go">(tensor(2.3030),</span>
<span class="go"> (tensor([0., 0.]),</span>
<span class="go">  tensor([6., 6.])))</span>
</pre></div>
</div>
<p>Note:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>This function is significantly slower than `vhp` due to backward mode AD constraints.
If your functions is twice continuously differentiable, then hvp = vhp.t(). So if you
know that your function satisfies this condition, you should use vhp instead that is
much faster with the current implementation.
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="locally-disabling-gradient-computation">
<span id="locally-disable-grad"></span><h2>Locally disabling gradient computation<a class="headerlink" href="#locally-disabling-gradient-computation" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="torch.autograd.no_grad">
<em class="property">class </em><code class="sig-prename descclassname">torch.autograd.</code><code class="sig-name descname">no_grad</code><a class="reference internal" href="_modules/torch/autograd/grad_mode.html#no_grad"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.no_grad" title="Permalink to this definition">¶</a></dt>
<dd><p>Context-manager that disabled gradient calculation.</p>
<p>Disabling gradient calculation is useful for inference, when you are sure
that you will not call <code class="xref py py-meth docutils literal notranslate"><span class="pre">Tensor.backward()</span></code>. It will reduce memory
consumption for computations that would otherwise have <cite>requires_grad=True</cite>.</p>
<p>In this mode, the result of every computation will have
<cite>requires_grad=False</cite>, even when the inputs have <cite>requires_grad=True</cite>.</p>
<p>This mode has no effect when using <a class="reference internal" href="#torch.autograd.enable_grad" title="torch.autograd.enable_grad"><code class="xref py py-class docutils literal notranslate"><span class="pre">enable_grad</span></code></a> context manager .</p>
<p>This context manager is thread local; it will not affect computation
in other threads.</p>
<p>Also functions as a decorator. (Make sure to instantiate with parenthesis.)</p>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">x</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">],</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
<span class="gp">... </span>  <span class="n">y</span> <span class="o">=</span> <span class="n">x</span> <span class="o">*</span> <span class="mi">2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="o">.</span><span class="n">requires_grad</span>
<span class="go">False</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nd">@torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">()</span>
<span class="gp">... </span><span class="k">def</span> <span class="nf">doubler</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
<span class="gp">... </span>    <span class="k">return</span> <span class="n">x</span> <span class="o">*</span> <span class="mi">2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">z</span> <span class="o">=</span> <span class="n">doubler</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">z</span><span class="o">.</span><span class="n">requires_grad</span>
<span class="go">False</span>
</pre></div>
</div>
</dd></dl>

<dl class="class">
<dt id="torch.autograd.enable_grad">
<em class="property">class </em><code class="sig-prename descclassname">torch.autograd.</code><code class="sig-name descname">enable_grad</code><a class="reference internal" href="_modules/torch/autograd/grad_mode.html#enable_grad"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.enable_grad" title="Permalink to this definition">¶</a></dt>
<dd><p>Context-manager that enables gradient calculation.</p>
<p>Enables gradient calculation, if it has been disabled via <a class="reference internal" href="#torch.autograd.no_grad" title="torch.autograd.no_grad"><code class="xref py py-class docutils literal notranslate"><span class="pre">no_grad</span></code></a>
or <a class="reference internal" href="#torch.autograd.set_grad_enabled" title="torch.autograd.set_grad_enabled"><code class="xref py py-class docutils literal notranslate"><span class="pre">set_grad_enabled</span></code></a>.</p>
<p>This context manager is thread local; it will not affect computation
in other threads.</p>
<p>Also functions as a decorator. (Make sure to instantiate with parenthesis.)</p>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">x</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">],</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
<span class="gp">... </span>  <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">enable_grad</span><span class="p">():</span>
<span class="gp">... </span>    <span class="n">y</span> <span class="o">=</span> <span class="n">x</span> <span class="o">*</span> <span class="mi">2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="o">.</span><span class="n">requires_grad</span>
<span class="go">True</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">x</span><span class="o">.</span><span class="n">grad</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nd">@torch</span><span class="o">.</span><span class="n">enable_grad</span><span class="p">()</span>
<span class="gp">... </span><span class="k">def</span> <span class="nf">doubler</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
<span class="gp">... </span>    <span class="k">return</span> <span class="n">x</span> <span class="o">*</span> <span class="mi">2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
<span class="gp">... </span>    <span class="n">z</span> <span class="o">=</span> <span class="n">doubler</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">z</span><span class="o">.</span><span class="n">requires_grad</span>
<span class="go">True</span>
</pre></div>
</div>
</dd></dl>

<dl class="class">
<dt id="torch.autograd.set_grad_enabled">
<em class="property">class </em><code class="sig-prename descclassname">torch.autograd.</code><code class="sig-name descname">set_grad_enabled</code><span class="sig-paren">(</span><em class="sig-param">mode</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/grad_mode.html#set_grad_enabled"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.set_grad_enabled" title="Permalink to this definition">¶</a></dt>
<dd><p>Context-manager that sets gradient calculation to on or off.</p>
<p><code class="docutils literal notranslate"><span class="pre">set_grad_enabled</span></code> will enable or disable grads based on its argument <code class="xref py py-attr docutils literal notranslate"><span class="pre">mode</span></code>.
It can be used as a context-manager or as a function.</p>
<p>When using <a class="reference internal" href="#torch.autograd.enable_grad" title="torch.autograd.enable_grad"><code class="xref py py-class docutils literal notranslate"><span class="pre">enable_grad</span></code></a> context manager, <code class="xref py py-class docutils literal notranslate"><span class="pre">set_grad_enabled(False)</span></code>
has no effect.</p>
<p>This context manager is thread local; it will not affect computation
in other threads.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>mode</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a>) – Flag whether to enable grad (<code class="docutils literal notranslate"><span class="pre">True</span></code>), or disable
(<code class="docutils literal notranslate"><span class="pre">False</span></code>). This can be used to conditionally enable
gradients.</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">x</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">],</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">is_train</span> <span class="o">=</span> <span class="kc">False</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">set_grad_enabled</span><span class="p">(</span><span class="n">is_train</span><span class="p">):</span>
<span class="gp">... </span>  <span class="n">y</span> <span class="o">=</span> <span class="n">x</span> <span class="o">*</span> <span class="mi">2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="o">.</span><span class="n">requires_grad</span>
<span class="go">False</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">torch</span><span class="o">.</span><span class="n">set_grad_enabled</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span> <span class="o">=</span> <span class="n">x</span> <span class="o">*</span> <span class="mi">2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="o">.</span><span class="n">requires_grad</span>
<span class="go">True</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">torch</span><span class="o">.</span><span class="n">set_grad_enabled</span><span class="p">(</span><span class="kc">False</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span> <span class="o">=</span> <span class="n">x</span> <span class="o">*</span> <span class="mi">2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="o">.</span><span class="n">requires_grad</span>
<span class="go">False</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="in-place-operations-on-tensors">
<h2>In-place operations on Tensors<a class="headerlink" href="#in-place-operations-on-tensors" title="Permalink to this headline">¶</a></h2>
<p>Supporting in-place operations in autograd is a hard matter, and we discourage
their use in most cases. Autograd’s aggressive buffer freeing and reuse makes
it very efficient and there are very few occasions when in-place operations
actually lower memory usage by any significant amount. Unless you’re operating
under heavy memory pressure, you might never need to use them.</p>
<div class="section" id="in-place-correctness-checks">
<h3>In-place correctness checks<a class="headerlink" href="#in-place-correctness-checks" title="Permalink to this headline">¶</a></h3>
<p>All <code class="xref py py-class docutils literal notranslate"><span class="pre">Tensor</span></code> s keep track of in-place operations applied to them, and
if the implementation detects that a tensor was saved for backward in one of
the functions, but it was modified in-place afterwards, an error will be raised
once backward pass is started. This ensures that if you’re using in-place
functions and not seeing any errors, you can be sure that the computed
gradients are correct.</p>
</div>
</div>
<div class="section" id="variable-deprecated">
<h2>Variable (deprecated)<a class="headerlink" href="#variable-deprecated" title="Permalink to this headline">¶</a></h2>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>The Variable API has been deprecated: Variables are no longer necessary to
use autograd with tensors. Autograd automatically supports Tensors with
<code class="docutils literal notranslate"><span class="pre">requires_grad</span></code> set to <code class="docutils literal notranslate"><span class="pre">True</span></code>. Below please find a quick guide on what
has changed:</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">Variable(tensor)</span></code> and <code class="docutils literal notranslate"><span class="pre">Variable(tensor,</span> <span class="pre">requires_grad)</span></code> still work as expected,
but they return Tensors instead of Variables.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">var.data</span></code> is the same thing as <code class="docutils literal notranslate"><span class="pre">tensor.data</span></code>.</p></li>
<li><p>Methods such as <code class="docutils literal notranslate"><span class="pre">var.backward(),</span> <span class="pre">var.detach(),</span> <span class="pre">var.register_hook()</span></code> now work on tensors
with the same method names.</p></li>
</ul>
<p>In addition, one can now create tensors with <code class="docutils literal notranslate"><span class="pre">requires_grad=True</span></code> using factory
methods such as <a class="reference internal" href="torch.html#torch.randn" title="torch.randn"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.randn()</span></code></a>, <a class="reference internal" href="torch.html#torch.zeros" title="torch.zeros"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.zeros()</span></code></a>, <a class="reference internal" href="torch.html#torch.ones" title="torch.ones"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.ones()</span></code></a>, and others
like the following:</p>
<p><code class="docutils literal notranslate"><span class="pre">autograd_tensor</span> <span class="pre">=</span> <span class="pre">torch.randn((2,</span> <span class="pre">3,</span> <span class="pre">4),</span> <span class="pre">requires_grad=True)</span></code></p>
</div>
</div>
<div class="section" id="tensor-autograd-functions">
<h2>Tensor autograd functions<a class="headerlink" href="#tensor-autograd-functions" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt>
<em class="property">class </em><code class="sig-prename descclassname">torch.</code><code class="sig-name descname">Tensor</code></dt>
<dd><dl class="attribute">
<dt id="torch.Tensor.grad">
<code class="sig-name descname">grad</code><a class="headerlink" href="#torch.Tensor.grad" title="Permalink to this definition">¶</a></dt>
<dd><p>This attribute is <code class="docutils literal notranslate"><span class="pre">None</span></code> by default and becomes a Tensor the first time a call to
<a class="reference internal" href="#torch.Tensor.backward" title="torch.Tensor.backward"><code class="xref py py-func docutils literal notranslate"><span class="pre">backward()</span></code></a> computes gradients for <code class="docutils literal notranslate"><span class="pre">self</span></code>.
The attribute will then contain the gradients computed and future calls to
<a class="reference internal" href="#torch.Tensor.backward" title="torch.Tensor.backward"><code class="xref py py-func docutils literal notranslate"><span class="pre">backward()</span></code></a> will accumulate (add) gradients into it.</p>
</dd></dl>

<dl class="attribute">
<dt id="torch.Tensor.requires_grad">
<code class="sig-name descname">requires_grad</code><a class="headerlink" href="#torch.Tensor.requires_grad" title="Permalink to this definition">¶</a></dt>
<dd><p>Is <code class="docutils literal notranslate"><span class="pre">True</span></code> if gradients need to be computed for this Tensor, <code class="docutils literal notranslate"><span class="pre">False</span></code> otherwise.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The fact that gradients need to be computed for a Tensor do not mean that the <a class="reference internal" href="#torch.Tensor.grad" title="torch.Tensor.grad"><code class="xref py py-attr docutils literal notranslate"><span class="pre">grad</span></code></a>
attribute will be populated, see <a class="reference internal" href="#torch.Tensor.is_leaf" title="torch.Tensor.is_leaf"><code class="xref py py-attr docutils literal notranslate"><span class="pre">is_leaf</span></code></a> for more details.</p>
</div>
</dd></dl>

<dl class="attribute">
<dt id="torch.Tensor.is_leaf">
<code class="sig-name descname">is_leaf</code><a class="headerlink" href="#torch.Tensor.is_leaf" title="Permalink to this definition">¶</a></dt>
<dd><p>All Tensors that have <a class="reference internal" href="#torch.Tensor.requires_grad" title="torch.Tensor.requires_grad"><code class="xref py py-attr docutils literal notranslate"><span class="pre">requires_grad</span></code></a> which is <code class="docutils literal notranslate"><span class="pre">False</span></code> will be leaf Tensors by convention.</p>
<p>For Tensors that have <a class="reference internal" href="#torch.Tensor.requires_grad" title="torch.Tensor.requires_grad"><code class="xref py py-attr docutils literal notranslate"><span class="pre">requires_grad</span></code></a> which is <code class="docutils literal notranslate"><span class="pre">True</span></code>, they will be leaf Tensors if they were
created by the user. This means that they are not the result of an operation and so
<code class="xref py py-attr docutils literal notranslate"><span class="pre">grad_fn</span></code> is None.</p>
<p>Only leaf Tensors will have their <a class="reference internal" href="#torch.Tensor.grad" title="torch.Tensor.grad"><code class="xref py py-attr docutils literal notranslate"><span class="pre">grad</span></code></a> populated during a call to <a class="reference internal" href="#torch.Tensor.backward" title="torch.Tensor.backward"><code class="xref py py-func docutils literal notranslate"><span class="pre">backward()</span></code></a>.
To get <a class="reference internal" href="#torch.Tensor.grad" title="torch.Tensor.grad"><code class="xref py py-attr docutils literal notranslate"><span class="pre">grad</span></code></a> populated for non-leaf Tensors, you can use <a class="reference internal" href="#torch.Tensor.retain_grad" title="torch.Tensor.retain_grad"><code class="xref py py-func docutils literal notranslate"><span class="pre">retain_grad()</span></code></a>.</p>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">a</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">a</span><span class="o">.</span><span class="n">is_leaf</span>
<span class="go">True</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">b</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">b</span><span class="o">.</span><span class="n">is_leaf</span>
<span class="go">False</span>
<span class="go"># b was created by the operation that cast a cpu Tensor into a cuda Tensor</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">c</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> <span class="o">+</span> <span class="mi">2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">c</span><span class="o">.</span><span class="n">is_leaf</span>
<span class="go">False</span>
<span class="go"># c was created by the addition operation</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">d</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">d</span><span class="o">.</span><span class="n">is_leaf</span>
<span class="go">True</span>
<span class="go"># d does not require gradients and so has no operation creating it (that is tracked by the autograd engine)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">e</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span><span class="o">.</span><span class="n">requires_grad_</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">e</span><span class="o">.</span><span class="n">is_leaf</span>
<span class="go">True</span>
<span class="go"># e requires gradients and has no operations creating it</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">f</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="s2">&quot;cuda&quot;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">f</span><span class="o">.</span><span class="n">is_leaf</span>
<span class="go">True</span>
<span class="go"># f requires grad, has no operation creating it</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.Tensor.backward">
<code class="sig-name descname">backward</code><span class="sig-paren">(</span><em class="sig-param">gradient=None</em>, <em class="sig-param">retain_graph=None</em>, <em class="sig-param">create_graph=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/tensor.html#Tensor.backward"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.Tensor.backward" title="Permalink to this definition">¶</a></dt>
<dd><p>Computes the gradient of current tensor w.r.t. graph leaves.</p>
<p>The graph is differentiated using the chain rule. If the tensor is
non-scalar (i.e. its data has more than one element) and requires
gradient, the function additionally requires specifying <code class="docutils literal notranslate"><span class="pre">gradient</span></code>.
It should be a tensor of matching type and location, that contains
the gradient of the differentiated function w.r.t. <code class="docutils literal notranslate"><span class="pre">self</span></code>.</p>
<p>This function accumulates gradients in the leaves - you might need to
zero them before calling it.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>gradient</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.8)"><em>None</em></a>) – Gradient w.r.t. the
tensor. If it is a tensor, it will be automatically converted
to a Tensor that does not require grad unless <code class="docutils literal notranslate"><span class="pre">create_graph</span></code> is True.
None values can be specified for scalar Tensors or ones that
don’t require grad. If a None value would be acceptable then
this argument is optional.</p></li>
<li><p><strong>retain_graph</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">False</span></code>, the graph used to compute
the grads will be freed. Note that in nearly all cases setting
this option to True is not needed and often can be worked around
in a much more efficient way. Defaults to the value of
<code class="docutils literal notranslate"><span class="pre">create_graph</span></code>.</p></li>
<li><p><strong>create_graph</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, graph of the derivative will
be constructed, allowing to compute higher order derivative
products. Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.Tensor.detach">
<code class="sig-name descname">detach</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#torch.Tensor.detach" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns a new Tensor, detached from the current graph.</p>
<p>The result will never require gradient.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Returned Tensor shares the same storage with the original one.
In-place modifications on either of them will be seen, and may trigger
errors in correctness checks.
IMPORTANT NOTE: Previously, in-place size / stride / storage changes
(such as <cite>resize_</cite> / <cite>resize_as_</cite> / <cite>set_</cite> / <cite>transpose_</cite>) to the returned tensor
also update the original tensor. Now, these in-place changes will not update the
original tensor anymore, and will instead trigger an error.
For sparse tensors:
In-place indices / values changes (such as <cite>zero_</cite> / <cite>copy_</cite> / <cite>add_</cite>) to the
returned tensor will not update the original tensor anymore, and will instead
trigger an error.</p>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.Tensor.detach_">
<code class="sig-name descname">detach_</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#torch.Tensor.detach_" title="Permalink to this definition">¶</a></dt>
<dd><p>Detaches the Tensor from the graph that created it, making it a leaf.
Views cannot be detached in-place.</p>
</dd></dl>

<dl class="method">
<dt id="torch.Tensor.register_hook">
<code class="sig-name descname">register_hook</code><span class="sig-paren">(</span><em class="sig-param">hook</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/tensor.html#Tensor.register_hook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.Tensor.register_hook" title="Permalink to this definition">¶</a></dt>
<dd><p>Registers a backward hook.</p>
<p>The hook will be called every time a gradient with respect to the
Tensor is computed. The hook should have the following signature:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">hook</span><span class="p">(</span><span class="n">grad</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span> <span class="ow">or</span> <span class="kc">None</span>
</pre></div>
</div>
<p>The hook should not modify its argument, but it can optionally return
a new gradient which will be used in place of <a class="reference internal" href="#torch.Tensor.grad" title="torch.Tensor.grad"><code class="xref py py-attr docutils literal notranslate"><span class="pre">grad</span></code></a>.</p>
<p>This function returns a handle with a method <code class="docutils literal notranslate"><span class="pre">handle.remove()</span></code>
that removes the hook from the module.</p>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">v</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.</span><span class="p">,</span> <span class="mf">0.</span><span class="p">,</span> <span class="mf">0.</span><span class="p">],</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">h</span> <span class="o">=</span> <span class="n">v</span><span class="o">.</span><span class="n">register_hook</span><span class="p">(</span><span class="k">lambda</span> <span class="n">grad</span><span class="p">:</span> <span class="n">grad</span> <span class="o">*</span> <span class="mi">2</span><span class="p">)</span>  <span class="c1"># double the gradient</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">v</span><span class="o">.</span><span class="n">backward</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">1.</span><span class="p">,</span> <span class="mf">2.</span><span class="p">,</span> <span class="mf">3.</span><span class="p">]))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">v</span><span class="o">.</span><span class="n">grad</span>

<span class="go"> 2</span>
<span class="go"> 4</span>
<span class="go"> 6</span>
<span class="go">[torch.FloatTensor of size (3,)]</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">h</span><span class="o">.</span><span class="n">remove</span><span class="p">()</span>  <span class="c1"># removes the hook</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.Tensor.retain_grad">
<code class="sig-name descname">retain_grad</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/tensor.html#Tensor.retain_grad"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.Tensor.retain_grad" title="Permalink to this definition">¶</a></dt>
<dd><p>Enables .grad attribute for non-leaf Tensors.</p>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="function">
<h2><span class="hidden-section">Function</span><a class="headerlink" href="#function" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="torch.autograd.Function">
<em class="property">class </em><code class="sig-prename descclassname">torch.autograd.</code><code class="sig-name descname">Function</code><a class="reference internal" href="_modules/torch/autograd/function.html#Function"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.Function" title="Permalink to this definition">¶</a></dt>
<dd><p>Records operation history and defines formulas for differentiating ops.</p>
<p>Every operation performed on <code class="xref py py-class docutils literal notranslate"><span class="pre">Tensor</span></code> s creates a new function
object, that performs the computation, and records that it happened.
The history is retained in the form of a DAG of functions, with edges
denoting data dependencies (<code class="docutils literal notranslate"><span class="pre">input</span> <span class="pre">&lt;-</span> <span class="pre">output</span></code>). Then, when backward is
called, the graph is processed in the topological ordering, by calling
<a class="reference internal" href="#torch.autograd.backward" title="torch.autograd.backward"><code class="xref py py-func docutils literal notranslate"><span class="pre">backward()</span></code></a> methods of each <a class="reference internal" href="#torch.autograd.Function" title="torch.autograd.Function"><code class="xref py py-class docutils literal notranslate"><span class="pre">Function</span></code></a> object, and passing
returned gradients on to next <a class="reference internal" href="#torch.autograd.Function" title="torch.autograd.Function"><code class="xref py py-class docutils literal notranslate"><span class="pre">Function</span></code></a> s.</p>
<p>Normally, the only way users interact with functions is by creating
subclasses and defining new operations. This is a recommended way of
extending torch.autograd.</p>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">class</span> <span class="nc">Exp</span><span class="p">(</span><span class="n">Function</span><span class="p">):</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="nd">@staticmethod</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="n">ctx</span><span class="p">,</span> <span class="n">i</span><span class="p">):</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="n">result</span> <span class="o">=</span> <span class="n">i</span><span class="o">.</span><span class="n">exp</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="n">ctx</span><span class="o">.</span><span class="n">save_for_backward</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="k">return</span> <span class="n">result</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="nd">@staticmethod</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="k">def</span> <span class="nf">backward</span><span class="p">(</span><span class="n">ctx</span><span class="p">,</span> <span class="n">grad_output</span><span class="p">):</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="n">result</span><span class="p">,</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">saved_tensors</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="k">return</span> <span class="n">grad_output</span> <span class="o">*</span> <span class="n">result</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1">#Use it by calling the apply method:</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">Exp</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
<dl class="method">
<dt id="torch.autograd.Function.backward">
<em class="property">static </em><code class="sig-name descname">backward</code><span class="sig-paren">(</span><em class="sig-param">ctx</em>, <em class="sig-param">*grad_outputs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/function.html#Function.backward"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.Function.backward" title="Permalink to this definition">¶</a></dt>
<dd><p>Defines a formula for differentiating the operation.</p>
<p>This function is to be overridden by all subclasses.</p>
<p>It must accept a context <code class="xref py py-attr docutils literal notranslate"><span class="pre">ctx</span></code> as the first argument, followed by
as many outputs did <a class="reference internal" href="#torch.autograd.Function.forward" title="torch.autograd.Function.forward"><code class="xref py py-func docutils literal notranslate"><span class="pre">forward()</span></code></a> return, and it should return as many
tensors, as there were inputs to <a class="reference internal" href="#torch.autograd.Function.forward" title="torch.autograd.Function.forward"><code class="xref py py-func docutils literal notranslate"><span class="pre">forward()</span></code></a>. Each argument is the
gradient w.r.t the given output, and each returned value should be the
gradient w.r.t. the corresponding input.</p>
<p>The context can be used to retrieve tensors saved during the forward
pass. It also has an attribute <code class="xref py py-attr docutils literal notranslate"><span class="pre">ctx.needs_input_grad</span></code> as a tuple
of booleans representing whether each input needs gradient. E.g.,
<a class="reference internal" href="#torch.autograd.backward" title="torch.autograd.backward"><code class="xref py py-func docutils literal notranslate"><span class="pre">backward()</span></code></a> will have <code class="docutils literal notranslate"><span class="pre">ctx.needs_input_grad[0]</span> <span class="pre">=</span> <span class="pre">True</span></code> if the
first input to <a class="reference internal" href="#torch.autograd.Function.forward" title="torch.autograd.Function.forward"><code class="xref py py-func docutils literal notranslate"><span class="pre">forward()</span></code></a> needs gradient computated w.r.t. the
output.</p>
</dd></dl>

<dl class="method">
<dt id="torch.autograd.Function.forward">
<em class="property">static </em><code class="sig-name descname">forward</code><span class="sig-paren">(</span><em class="sig-param">ctx</em>, <em class="sig-param">*args</em>, <em class="sig-param">**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/function.html#Function.forward"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.Function.forward" title="Permalink to this definition">¶</a></dt>
<dd><p>Performs the operation.</p>
<p>This function is to be overridden by all subclasses.</p>
<p>It must accept a context ctx as the first argument, followed by any
number of arguments (tensors or other types).</p>
<p>The context can be used to store tensors that can be then retrieved
during the backward pass.</p>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="context-method-mixins">
<h2>Context method mixins<a class="headerlink" href="#context-method-mixins" title="Permalink to this headline">¶</a></h2>
<p>When creating a new <a class="reference internal" href="#torch.autograd.Function" title="torch.autograd.Function"><code class="xref py py-class docutils literal notranslate"><span class="pre">Function</span></code></a>, the following methods are available to <cite>ctx</cite>.</p>
<dl class="class">
<dt id="torch.autograd.function._ContextMethodMixin">
<em class="property">class </em><code class="sig-prename descclassname">torch.autograd.function.</code><code class="sig-name descname">_ContextMethodMixin</code><a class="reference internal" href="_modules/torch/autograd/function.html#_ContextMethodMixin"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.function._ContextMethodMixin" title="Permalink to this definition">¶</a></dt>
<dd><dl class="method">
<dt id="torch.autograd.function._ContextMethodMixin.mark_dirty">
<code class="sig-name descname">mark_dirty</code><span class="sig-paren">(</span><em class="sig-param">*args</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/function.html#_ContextMethodMixin.mark_dirty"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.function._ContextMethodMixin.mark_dirty" title="Permalink to this definition">¶</a></dt>
<dd><p>Marks given tensors as modified in an in-place operation.</p>
<p><strong>This should be called at most once, only from inside the</strong>
<code class="xref py py-func docutils literal notranslate"><span class="pre">forward()</span></code> <strong>method, and all arguments should be inputs.</strong></p>
<p>Every tensor that’s been modified in-place in a call to <code class="xref py py-func docutils literal notranslate"><span class="pre">forward()</span></code>
should be given to this function, to ensure correctness of our checks.
It doesn’t matter whether the function is called before or after
modification.</p>
</dd></dl>

<dl class="method">
<dt id="torch.autograd.function._ContextMethodMixin.mark_non_differentiable">
<code class="sig-name descname">mark_non_differentiable</code><span class="sig-paren">(</span><em class="sig-param">*args</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/function.html#_ContextMethodMixin.mark_non_differentiable"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.function._ContextMethodMixin.mark_non_differentiable" title="Permalink to this definition">¶</a></dt>
<dd><p>Marks outputs as non-differentiable.</p>
<p><strong>This should be called at most once, only from inside the</strong>
<code class="xref py py-func docutils literal notranslate"><span class="pre">forward()</span></code> <strong>method, and all arguments should be outputs.</strong></p>
<p>This will mark outputs as not requiring gradients, increasing the
efficiency of backward computation. You still need to accept a gradient
for each output in <code class="xref py py-meth docutils literal notranslate"><span class="pre">backward()</span></code>, but it’s always going to
be a zero tensor with the same shape as the shape of a corresponding
output.</p>
<p>This is used e.g. for indices returned from a max <code class="xref py py-class docutils literal notranslate"><span class="pre">Function</span></code>.</p>
</dd></dl>

<dl class="method">
<dt id="torch.autograd.function._ContextMethodMixin.save_for_backward">
<code class="sig-name descname">save_for_backward</code><span class="sig-paren">(</span><em class="sig-param">*tensors</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/function.html#_ContextMethodMixin.save_for_backward"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.function._ContextMethodMixin.save_for_backward" title="Permalink to this definition">¶</a></dt>
<dd><p>Saves given tensors for a future call to <code class="xref py py-func docutils literal notranslate"><span class="pre">backward()</span></code>.</p>
<p><strong>This should be called at most once, and only from inside the</strong>
<code class="xref py py-func docutils literal notranslate"><span class="pre">forward()</span></code> <strong>method.</strong></p>
<p>Later, saved tensors can be accessed through the <code class="xref py py-attr docutils literal notranslate"><span class="pre">saved_tensors</span></code>
attribute. Before returning them to the user, a check is made to ensure
they weren’t used in any in-place operation that modified their content.</p>
<p>Arguments can also be <code class="docutils literal notranslate"><span class="pre">None</span></code>.</p>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="numerical-gradient-checking">
<span id="grad-check"></span><h2>Numerical gradient checking<a class="headerlink" href="#numerical-gradient-checking" title="Permalink to this headline">¶</a></h2>
<dl class="function">
<dt id="torch.autograd.gradcheck">
<code class="sig-prename descclassname">torch.autograd.</code><code class="sig-name descname">gradcheck</code><span class="sig-paren">(</span><em class="sig-param">func</em>, <em class="sig-param">inputs</em>, <em class="sig-param">eps=1e-06</em>, <em class="sig-param">atol=1e-05</em>, <em class="sig-param">rtol=0.001</em>, <em class="sig-param">raise_exception=True</em>, <em class="sig-param">check_sparse_nnz=False</em>, <em class="sig-param">nondet_tol=0.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/gradcheck.html#gradcheck"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.gradcheck" title="Permalink to this definition">¶</a></dt>
<dd><p>Check gradients computed via small finite differences against analytical
gradients w.r.t. tensors in <code class="xref py py-attr docutils literal notranslate"><span class="pre">inputs</span></code> that are of floating point type
and with <code class="docutils literal notranslate"><span class="pre">requires_grad=True</span></code>.</p>
<p>The check between numerical and analytical gradients uses <a class="reference internal" href="torch.html#torch.allclose" title="torch.allclose"><code class="xref py py-func docutils literal notranslate"><span class="pre">allclose()</span></code></a>.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The default values are designed for <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> of double precision.
This check will likely fail if <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> is of less precision, e.g.,
<code class="docutils literal notranslate"><span class="pre">FloatTensor</span></code>.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>If any checked tensor in <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> has overlapping memory, i.e.,
different indices pointing to the same memory address (e.g., from
<code class="xref py py-func docutils literal notranslate"><span class="pre">torch.expand()</span></code>), this check will likely fail because the numerical
gradients computed by point perturbation at such indices will change
values at all other indices that share the same memory address.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>func</strong> (<em>function</em>) – a Python function that takes Tensor inputs and returns
a Tensor or a tuple of Tensors</p></li>
<li><p><strong>inputs</strong> (<em>tuple of Tensor</em><em> or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – inputs to the function</p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.8)"><em>float</em></a><em>, </em><em>optional</em>) – perturbation for finite differences</p></li>
<li><p><strong>atol</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.8)"><em>float</em></a><em>, </em><em>optional</em>) – absolute tolerance</p></li>
<li><p><strong>rtol</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.8)"><em>float</em></a><em>, </em><em>optional</em>) – relative tolerance</p></li>
<li><p><strong>raise_exception</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – indicating whether to raise an exception if
the check fails. The exception gives more information about the
exact nature of the failure. This is helpful when debugging gradchecks.</p></li>
<li><p><strong>check_sparse_nnz</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – if True, gradcheck allows for SparseTensor input,
and for any SparseTensor at input, gradcheck will perform check at nnz positions only.</p></li>
<li><p><strong>nondet_tol</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.8)"><em>float</em></a><em>, </em><em>optional</em>) – tolerance for non-determinism. When running
identical inputs through the differentiation, the results must either match
exactly (default, 0.0) or be within this tolerance.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>True if all differences satisfy allclose condition</p>
</dd>
</dl>
</dd></dl>

<dl class="function">
<dt id="torch.autograd.gradgradcheck">
<code class="sig-prename descclassname">torch.autograd.</code><code class="sig-name descname">gradgradcheck</code><span class="sig-paren">(</span><em class="sig-param">func</em>, <em class="sig-param">inputs</em>, <em class="sig-param">grad_outputs=None</em>, <em class="sig-param">eps=1e-06</em>, <em class="sig-param">atol=1e-05</em>, <em class="sig-param">rtol=0.001</em>, <em class="sig-param">gen_non_contig_grad_outputs=False</em>, <em class="sig-param">raise_exception=True</em>, <em class="sig-param">nondet_tol=0.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/gradcheck.html#gradgradcheck"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.gradgradcheck" title="Permalink to this definition">¶</a></dt>
<dd><p>Check gradients of gradients computed via small finite differences
against analytical gradients w.r.t. tensors in <code class="xref py py-attr docutils literal notranslate"><span class="pre">inputs</span></code> and
<code class="xref py py-attr docutils literal notranslate"><span class="pre">grad_outputs</span></code> that are of floating point type and with
<code class="docutils literal notranslate"><span class="pre">requires_grad=True</span></code>.</p>
<p>This function checks that backpropagating through the gradients computed
to the given <code class="xref py py-attr docutils literal notranslate"><span class="pre">grad_outputs</span></code> are correct.</p>
<p>The check between numerical and analytical gradients uses <a class="reference internal" href="torch.html#torch.allclose" title="torch.allclose"><code class="xref py py-func docutils literal notranslate"><span class="pre">allclose()</span></code></a>.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The default values are designed for <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> and
<code class="xref py py-attr docutils literal notranslate"><span class="pre">grad_outputs</span></code> of double precision. This check will likely fail if
they are of less precision, e.g., <code class="docutils literal notranslate"><span class="pre">FloatTensor</span></code>.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>If any checked tensor in <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">grad_outputs</span></code> has
overlapping memory, i.e., different indices pointing to the same memory
address (e.g., from <code class="xref py py-func docutils literal notranslate"><span class="pre">torch.expand()</span></code>), this check will likely fail
because the numerical gradients computed by point perturbation at such
indices will change values at all other indices that share the same
memory address.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>func</strong> (<em>function</em>) – a Python function that takes Tensor inputs and returns
a Tensor or a tuple of Tensors</p></li>
<li><p><strong>inputs</strong> (<em>tuple of Tensor</em><em> or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – inputs to the function</p></li>
<li><p><strong>grad_outputs</strong> (<em>tuple of Tensor</em><em> or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – The gradients with
respect to the function’s outputs.</p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.8)"><em>float</em></a><em>, </em><em>optional</em>) – perturbation for finite differences</p></li>
<li><p><strong>atol</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.8)"><em>float</em></a><em>, </em><em>optional</em>) – absolute tolerance</p></li>
<li><p><strong>rtol</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.8)"><em>float</em></a><em>, </em><em>optional</em>) – relative tolerance</p></li>
<li><p><strong>gen_non_contig_grad_outputs</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – if <code class="xref py py-attr docutils literal notranslate"><span class="pre">grad_outputs</span></code> is
<code class="docutils literal notranslate"><span class="pre">None</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">gen_non_contig_grad_outputs</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, the
randomly generated gradient outputs are made to be noncontiguous</p></li>
<li><p><strong>raise_exception</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – indicating whether to raise an exception if
the check fails. The exception gives more information about the
exact nature of the failure. This is helpful when debugging gradchecks.</p></li>
<li><p><strong>nondet_tol</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.8)"><em>float</em></a><em>, </em><em>optional</em>) – tolerance for non-determinism. When running
identical inputs through the differentiation, the results must either match
exactly (default, 0.0) or be within this tolerance. Note that a small amount
of nondeterminism in the gradient will lead to larger inaccuracies in
the second derivative.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>True if all differences satisfy allclose condition</p>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="profiler">
<h2>Profiler<a class="headerlink" href="#profiler" title="Permalink to this headline">¶</a></h2>
<p>Autograd includes a profiler that lets you inspect the cost of different
operators inside your model - both on the CPU and GPU. There are two modes
implemented at the moment - CPU-only using <a class="reference internal" href="#torch.autograd.profiler.profile" title="torch.autograd.profiler.profile"><code class="xref py py-class docutils literal notranslate"><span class="pre">profile</span></code></a>.
and nvprof based (registers both CPU and GPU activity) using
<a class="reference internal" href="#torch.autograd.profiler.emit_nvtx" title="torch.autograd.profiler.emit_nvtx"><code class="xref py py-class docutils literal notranslate"><span class="pre">emit_nvtx</span></code></a>.</p>
<dl class="class">
<dt id="torch.autograd.profiler.profile">
<em class="property">class </em><code class="sig-prename descclassname">torch.autograd.profiler.</code><code class="sig-name descname">profile</code><span class="sig-paren">(</span><em class="sig-param">enabled=True</em>, <em class="sig-param">use_cuda=False</em>, <em class="sig-param">record_shapes=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/profiler.html#profile"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.profiler.profile" title="Permalink to this definition">¶</a></dt>
<dd><p>Context manager that manages autograd profiler state and holds a summary of results.
Under the hood it just records events of functions being executed in C++ and
exposes those events to Python. You can wrap any code into it and it will
only report runtime of PyTorch functions.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>enabled</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – Setting this to False makes this context manager a no-op.
Default: <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p></li>
<li><p><strong>use_cuda</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – Enables timing of CUDA events as well using the cudaEvent API.
Adds approximately 4us of overhead to each tensor operation.
Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
<li><p><strong>record_shapes</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If shapes recording is set, information
about input dimensions will be collected. This allows one to see which
dimensions have been used under the hood and further group by them
using prof.key_averages(group_by_input_shape=True). Please note that
shape recording might skew your profiling data. It is recommended to
use separate runs with and without shape recording to validate the timing.
Most likely the skew will be negligible for bottom most events (in a case
of nested function calls). But for higher level functions the total
self cpu time might be artificially increased because of the shape
collection.</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Example</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">x</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">((</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">profiler</span><span class="o">.</span><span class="n">profile</span><span class="p">()</span> <span class="k">as</span> <span class="n">prof</span><span class="p">:</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">100</span><span class="p">):</span>  <span class="c1"># any normal python code, really!</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="n">y</span> <span class="o">=</span> <span class="n">x</span> <span class="o">**</span> <span class="mi">2</span>
<span class="go">&gt;&gt;          y.backward()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># NOTE: some columns were removed for brevity</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">prof</span><span class="o">.</span><span class="n">key_averages</span><span class="p">()</span><span class="o">.</span><span class="n">table</span><span class="p">(</span><span class="n">sort_by</span><span class="o">=</span><span class="s2">&quot;self_cpu_time_total&quot;</span><span class="p">))</span>
<span class="go">-----------------------------------  ---------------  ---------------  ---------------</span>
<span class="go">Name                                 Self CPU total   CPU time avg     Number of Calls</span>
<span class="go">-----------------------------------  ---------------  ---------------  ---------------</span>
<span class="go">mul                                  32.048ms         32.048ms         200</span>
<span class="go">pow                                  27.041ms         27.041ms         200</span>
<span class="go">PowBackward0                         9.727ms          55.483ms         100</span>
<span class="go">torch::autograd::AccumulateGrad      9.148ms          9.148ms          100</span>
<span class="go">torch::autograd::GraphRoot           691.816us        691.816us        100</span>
<span class="go">-----------------------------------  ---------------  ---------------  ---------------</span>
</pre></div>
</div>
<dl class="method">
<dt id="torch.autograd.profiler.profile.export_chrome_trace">
<code class="sig-name descname">export_chrome_trace</code><span class="sig-paren">(</span><em class="sig-param">path</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/profiler.html#profile.export_chrome_trace"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.profiler.profile.export_chrome_trace" title="Permalink to this definition">¶</a></dt>
<dd><p>Exports an EventList as a Chrome tracing tools file.</p>
<p>The checkpoint can be later loaded and inspected under <code class="docutils literal notranslate"><span class="pre">chrome://tracing</span></code> URL.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>path</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – Path where the trace will be written.</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.autograd.profiler.profile.key_averages">
<code class="sig-name descname">key_averages</code><span class="sig-paren">(</span><em class="sig-param">group_by_input_shape=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/profiler.html#profile.key_averages"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.profiler.profile.key_averages" title="Permalink to this definition">¶</a></dt>
<dd><p>Averages all function events over their keys.</p>
<p>&#64;param group_by_input_shapes The key would become
(event name, input dimensions) rather than just event name.
This is useful to see which dimensionality contributes to the runtime
the most and may help with dimension specific optimizations or
choosing best candidates for quantization (aka fitting a roof line)</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>An EventList containing FunctionEventAvg objects.</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.autograd.profiler.profile.self_cpu_time_total">
<em class="property">property </em><code class="sig-name descname">self_cpu_time_total</code><a class="headerlink" href="#torch.autograd.profiler.profile.self_cpu_time_total" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns total time spent on CPU obtained as a sum of
all self times across all the events.</p>
</dd></dl>

<dl class="method">
<dt id="torch.autograd.profiler.profile.table">
<code class="sig-name descname">table</code><span class="sig-paren">(</span><em class="sig-param">sort_by=None</em>, <em class="sig-param">row_limit=100</em>, <em class="sig-param">header=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/profiler.html#profile.table"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.profiler.profile.table" title="Permalink to this definition">¶</a></dt>
<dd><p>Prints an EventList as a nicely formatted table.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>sort_by</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>, </em><em>optional</em>) – Attribute used to sort entries. By default
they are printed in the same order as they were registered.
Valid keys include: <code class="docutils literal notranslate"><span class="pre">cpu_time</span></code>, <code class="docutils literal notranslate"><span class="pre">cuda_time</span></code>, <code class="docutils literal notranslate"><span class="pre">cpu_time_total</span></code>,
<code class="docutils literal notranslate"><span class="pre">cuda_time_total</span></code>, <code class="docutils literal notranslate"><span class="pre">count</span></code>.</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>A string containing the table.</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.autograd.profiler.profile.total_average">
<code class="sig-name descname">total_average</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/profiler.html#profile.total_average"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.profiler.profile.total_average" title="Permalink to this definition">¶</a></dt>
<dd><p>Averages all events.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>A FunctionEventAvg object.</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="torch.autograd.profiler.record_function">
<em class="property">class </em><code class="sig-prename descclassname">torch.autograd.profiler.</code><code class="sig-name descname">record_function</code><span class="sig-paren">(</span><em class="sig-param">name</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/profiler.html#record_function"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.profiler.record_function" title="Permalink to this definition">¶</a></dt>
<dd><p>Context manager/function decorator that adds a label to a block of
Python code (or function) when running autograd profiler. It is
useful when tracing the code profile.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>name</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – Label assigned to the block of code.</p>
</dd>
</dl>
<p class="rubric">Example</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">x</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">((</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">profiler</span><span class="o">.</span><span class="n">profile</span><span class="p">()</span> <span class="k">as</span> <span class="n">prof</span><span class="p">:</span>
<span class="gp">... </span>    <span class="n">y</span> <span class="o">=</span> <span class="n">x</span> <span class="o">**</span> <span class="mi">2</span>
<span class="gp">... </span>    <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">profiler</span><span class="o">.</span><span class="n">record_function</span><span class="p">(</span><span class="s2">&quot;label-z&quot;</span><span class="p">):</span> <span class="c1"># label the block</span>
<span class="gp">... </span>        <span class="n">z</span> <span class="o">=</span> <span class="n">y</span> <span class="o">**</span> <span class="mi">3</span>
<span class="gp">... </span>    <span class="n">y</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
<span class="gp">...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># NOTE: some columns were removed for brevity</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">prof</span><span class="o">.</span><span class="n">key_averages</span><span class="p">()</span><span class="o">.</span><span class="n">table</span><span class="p">(</span><span class="n">sort_by</span><span class="o">=</span><span class="s2">&quot;self_cpu_time_total&quot;</span><span class="p">))</span>
<span class="go">-----------------------------------  ---------------  ---------------  ---------------</span>
<span class="go">Name                                 Self CPU total %  CPU time avg     Number of Calls</span>
<span class="go">-----------------------------------  ---------------  ---------------  ---------------</span>
<span class="go">pow                                  60.77%           47.470us         3</span>
<span class="go">mul                                  21.73%           25.465us         2</span>
<span class="go">PowBackward0                         12.03%           121.891us        1</span>
<span class="go">torch::autograd::AccumulateGrad      2.70%            6.324us          1</span>
<span class="go">label-z                              2.13%            12.421us         1</span>
<span class="go">torch::autograd::GraphRoot           0.64%            1.503us          1</span>
<span class="go">-----------------------------------  ---------------  ---------------  ---------------</span>
<span class="go">Self CPU time total: 234.344us</span>
<span class="go">CUDA time total: 0.000us</span>
</pre></div>
</div>
</dd></dl>

<dl class="class">
<dt id="torch.autograd.profiler.emit_nvtx">
<em class="property">class </em><code class="sig-prename descclassname">torch.autograd.profiler.</code><code class="sig-name descname">emit_nvtx</code><span class="sig-paren">(</span><em class="sig-param">enabled=True</em>, <em class="sig-param">record_shapes=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/profiler.html#emit_nvtx"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.profiler.emit_nvtx" title="Permalink to this definition">¶</a></dt>
<dd><p>Context manager that makes every autograd operation emit an NVTX range.</p>
<p>It is useful when running the program under nvprof:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">nvprof</span> <span class="o">--</span><span class="n">profile</span><span class="o">-</span><span class="n">from</span><span class="o">-</span><span class="n">start</span> <span class="n">off</span> <span class="o">-</span><span class="n">o</span> <span class="n">trace_name</span><span class="o">.</span><span class="n">prof</span> <span class="o">--</span> <span class="o">&lt;</span><span class="n">regular</span> <span class="n">command</span> <span class="n">here</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>Unfortunately, there’s no way to force nvprof to flush the data it collected
to disk, so for CUDA profiling one has to use this context manager to annotate
nvprof traces and wait for the process to exit before inspecting them.
Then, either NVIDIA Visual Profiler (nvvp) can be used to visualize the timeline, or
<a class="reference internal" href="#torch.autograd.profiler.load_nvprof" title="torch.autograd.profiler.load_nvprof"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.autograd.profiler.load_nvprof()</span></code></a> can load the results for inspection
e.g. in Python REPL.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>enabled</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em><em>, </em><em>default=True</em>) – Setting <code class="docutils literal notranslate"><span class="pre">enabled=False</span></code> makes this context manager a no-op.
Default: <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p></li>
<li><p><strong>record_shapes</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em><em>, </em><em>default=False</em>) – If <code class="docutils literal notranslate"><span class="pre">record_shapes=True</span></code>, the nvtx range wrapping
each autograd op will append information about the sizes of Tensor arguments received
by that op, in the following format:
<code class="docutils literal notranslate"><span class="pre">[[arg0.size(0),</span> <span class="pre">arg0.size(1),</span> <span class="pre">...],</span> <span class="pre">[arg1.size(0),</span> <span class="pre">arg1.size(1),</span> <span class="pre">...],</span> <span class="pre">...]</span></code>
Non-tensor arguments will be represented by <code class="docutils literal notranslate"><span class="pre">[]</span></code>.
Arguments will be listed in the order they are received by the backend op.
Please note that this order may not match the order in which those arguments were passed
on the Python side.  Also note that shape recording may increase the overhead of nvtx range creation.</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Example</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">profiler</span><span class="o">.</span><span class="n">profile</span><span class="p">():</span>
<span class="gp">... </span>    <span class="n">model</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="c1"># Warmup CUDA memory allocator and profiler</span>
<span class="gp">... </span>    <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">profiler</span><span class="o">.</span><span class="n">emit_nvtx</span><span class="p">():</span>
<span class="gp">... </span>        <span class="n">model</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
</pre></div>
</div>
<p><strong>Forward-backward correlation</strong></p>
<p>When viewing a profile created using <a class="reference internal" href="#torch.autograd.profiler.emit_nvtx" title="torch.autograd.profiler.emit_nvtx"><code class="xref py py-class docutils literal notranslate"><span class="pre">emit_nvtx</span></code></a> in the Nvidia Visual Profiler,
correlating each backward-pass op with the corresponding forward-pass op can be difficult.
To ease this task, <a class="reference internal" href="#torch.autograd.profiler.emit_nvtx" title="torch.autograd.profiler.emit_nvtx"><code class="xref py py-class docutils literal notranslate"><span class="pre">emit_nvtx</span></code></a> appends sequence number information to the ranges it
generates.</p>
<p>During the forward pass, each function range is decorated with <code class="docutils literal notranslate"><span class="pre">seq=&lt;N&gt;</span></code>.  <code class="docutils literal notranslate"><span class="pre">seq</span></code> is a running
counter, incremented each time a new backward Function object is created and stashed for backward.
Thus, the <code class="docutils literal notranslate"><span class="pre">seq=&lt;N&gt;</span></code> annotation associated with each forward function range tells you that
if a backward Function object is created by this forward function,
the backward object will receive sequence number N.
During the backward pass, the top-level range wrapping each C++ backward Function’s
<code class="docutils literal notranslate"><span class="pre">apply()</span></code> call is decorated with <code class="docutils literal notranslate"><span class="pre">stashed</span> <span class="pre">seq=&lt;M&gt;</span></code>.  <code class="docutils literal notranslate"><span class="pre">M</span></code> is the sequence number that
the backward object was created with.  By comparing <code class="docutils literal notranslate"><span class="pre">stashed</span> <span class="pre">seq</span></code> numbers in backward with <code class="docutils literal notranslate"><span class="pre">seq</span></code>
numbers in forward, you can track down which forward op created each backward Function.</p>
<p>Any functions executed during the backward pass are also decorated with <code class="docutils literal notranslate"><span class="pre">seq=&lt;N&gt;</span></code>.  During
default backward (with <code class="docutils literal notranslate"><span class="pre">create_graph=False</span></code>) this information is irrelevant, and in fact,
<code class="docutils literal notranslate"><span class="pre">N</span></code> may simply be 0 for all such functions.  Only the top-level ranges associated with
backward Function objects’ <code class="docutils literal notranslate"><span class="pre">apply()</span></code> methods are useful, as a way to correlate these Function
objects with the earlier forward pass.</p>
<p><strong>Double-backward</strong></p>
<p>If, on the other hand, a backward pass with <code class="docutils literal notranslate"><span class="pre">create_graph=True</span></code> is underway (in other words,
if you are setting up for a double-backward), each function’s execution during backward
is given a nonzero, useful <code class="docutils literal notranslate"><span class="pre">seq=&lt;N&gt;</span></code>.  Those functions may themselves create Function objects
to be executed later during double-backward, just as the original functions in the forward pass did.
The relationship between backward and double-backward is conceptually the same as the relationship
between forward and backward: The functions still emit current-sequence-number-tagged ranges,
the Function objects they create still stash those sequence numbers, and during the eventual
double-backward, the Function objects’ <code class="docutils literal notranslate"><span class="pre">apply()</span></code> ranges are still tagged with <code class="docutils literal notranslate"><span class="pre">stashed</span> <span class="pre">seq</span></code>
numbers, which can be compared to <cite>seq</cite> numbers from the backward pass.</p>
</dd></dl>

<dl class="function">
<dt id="torch.autograd.profiler.load_nvprof">
<code class="sig-prename descclassname">torch.autograd.profiler.</code><code class="sig-name descname">load_nvprof</code><span class="sig-paren">(</span><em class="sig-param">path</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/profiler.html#load_nvprof"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.profiler.load_nvprof" title="Permalink to this definition">¶</a></dt>
<dd><p>Opens an nvprof trace file and parses autograd annotations.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>path</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – path to nvprof trace</p>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="anomaly-detection">
<h2>Anomaly detection<a class="headerlink" href="#anomaly-detection" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="torch.autograd.detect_anomaly">
<em class="property">class </em><code class="sig-prename descclassname">torch.autograd.</code><code class="sig-name descname">detect_anomaly</code><a class="reference internal" href="_modules/torch/autograd/anomaly_mode.html#detect_anomaly"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.detect_anomaly" title="Permalink to this definition">¶</a></dt>
<dd><p>Context-manager that enable anomaly detection for the autograd engine.</p>
<p>This does two things:
- Running the forward pass with detection enabled will allow the backward
pass to print the traceback of the forward operation that created the failing
backward function.
- Any backward computation that generate “nan” value will raise an error.</p>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This mode should be enabled only for debugging as the different tests
will slow down your program execution.</p>
</div>
<p class="rubric">Example</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">torch</span> <span class="kn">import</span> <span class="n">autograd</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">class</span> <span class="nc">MyFunc</span><span class="p">(</span><span class="n">autograd</span><span class="o">.</span><span class="n">Function</span><span class="p">):</span>
<span class="gp">... </span>    <span class="nd">@staticmethod</span>
<span class="gp">... </span>    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="n">ctx</span><span class="p">,</span> <span class="n">inp</span><span class="p">):</span>
<span class="gp">... </span>        <span class="k">return</span> <span class="n">inp</span><span class="o">.</span><span class="n">clone</span><span class="p">()</span>
<span class="gp">... </span>    <span class="nd">@staticmethod</span>
<span class="gp">... </span>    <span class="k">def</span> <span class="nf">backward</span><span class="p">(</span><span class="n">ctx</span><span class="p">,</span> <span class="n">gO</span><span class="p">):</span>
<span class="gp">... </span>        <span class="c1"># Error during the backward pass</span>
<span class="gp">... </span>        <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;Some error in backward&quot;</span><span class="p">)</span>
<span class="gp">... </span>        <span class="k">return</span> <span class="n">gO</span><span class="o">.</span><span class="n">clone</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">run_fn</span><span class="p">(</span><span class="n">a</span><span class="p">):</span>
<span class="gp">... </span>    <span class="n">out</span> <span class="o">=</span> <span class="n">MyFunc</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="n">a</span><span class="p">)</span>
<span class="gp">... </span>    <span class="k">return</span> <span class="n">out</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">run_fn</span><span class="p">(</span><span class="n">inp</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">out</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
<span class="go">    Traceback (most recent call last):</span>
<span class="go">      File &quot;&lt;stdin&gt;&quot;, line 1, in &lt;module&gt;</span>
<span class="go">      File &quot;/your/pytorch/install/torch/tensor.py&quot;, line 93, in backward</span>
<span class="go">        torch.autograd.backward(self, gradient, retain_graph, create_graph)</span>
<span class="go">      File &quot;/your/pytorch/install/torch/autograd/__init__.py&quot;, line 90, in backward</span>
<span class="go">        allow_unreachable=True)  # allow_unreachable flag</span>
<span class="go">      File &quot;/your/pytorch/install/torch/autograd/function.py&quot;, line 76, in apply</span>
<span class="go">        return self._forward_cls.backward(self, *args)</span>
<span class="go">      File &quot;&lt;stdin&gt;&quot;, line 8, in backward</span>
<span class="go">    RuntimeError: Some error in backward</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">with</span> <span class="n">autograd</span><span class="o">.</span><span class="n">detect_anomaly</span><span class="p">():</span>
<span class="gp">... </span>    <span class="n">inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">... </span>    <span class="n">out</span> <span class="o">=</span> <span class="n">run_fn</span><span class="p">(</span><span class="n">inp</span><span class="p">)</span>
<span class="gp">... </span>    <span class="n">out</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
<span class="go">    Traceback of forward call that caused the error:</span>
<span class="go">      File &quot;tmp.py&quot;, line 53, in &lt;module&gt;</span>
<span class="go">        out = run_fn(inp)</span>
<span class="go">      File &quot;tmp.py&quot;, line 44, in run_fn</span>
<span class="go">        out = MyFunc.apply(a)</span>
<span class="go">    Traceback (most recent call last):</span>
<span class="go">      File &quot;&lt;stdin&gt;&quot;, line 4, in &lt;module&gt;</span>
<span class="go">      File &quot;/your/pytorch/install/torch/tensor.py&quot;, line 93, in backward</span>
<span class="go">        torch.autograd.backward(self, gradient, retain_graph, create_graph)</span>
<span class="go">      File &quot;/your/pytorch/install/torch/autograd/__init__.py&quot;, line 90, in backward</span>
<span class="go">        allow_unreachable=True)  # allow_unreachable flag</span>
<span class="go">      File &quot;/your/pytorch/install/torch/autograd/function.py&quot;, line 76, in apply</span>
<span class="go">        return self._forward_cls.backward(self, *args)</span>
<span class="go">      File &quot;&lt;stdin&gt;&quot;, line 8, in backward</span>
<span class="go">    RuntimeError: Some error in backward</span>
</pre></div>
</div>
</dd></dl>

<dl class="class">
<dt id="torch.autograd.set_detect_anomaly">
<em class="property">class </em><code class="sig-prename descclassname">torch.autograd.</code><code class="sig-name descname">set_detect_anomaly</code><span class="sig-paren">(</span><em class="sig-param">mode</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/anomaly_mode.html#set_detect_anomaly"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.autograd.set_detect_anomaly" title="Permalink to this definition">¶</a></dt>
<dd><p>Context-manager that sets the anomaly detection for the autograd engine on or off.</p>
<p><code class="docutils literal notranslate"><span class="pre">set_detect_anomaly</span></code> will enable or disable the autograd anomaly detection
based on its argument <code class="xref py py-attr docutils literal notranslate"><span class="pre">mode</span></code>.
It can be used as a context-manager or as a function.</p>
<p>See <code class="docutils literal notranslate"><span class="pre">detect_anomaly</span></code> above for details of the anomaly detection behaviour.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>mode</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a>) – Flag whether to enable anomaly detection (<code class="docutils literal notranslate"><span class="pre">True</span></code>),
or disable (<code class="docutils literal notranslate"><span class="pre">False</span></code>).</p>
</dd>
</dl>
</dd></dl>

</div>
</div>


             </article>
             
            </div>
            <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="cuda.html" class="btn btn-neutral float-right" title="torch.cuda" accesskey="n" rel="next">Next <img src="_static/images/chevron-right-orange.svg" class="next-page"></a>
      
      
        <a href="tensor_view.html" class="btn btn-neutral" title="Tensor Views" accesskey="p" rel="prev"><img src="_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>
      
    </div>
  

  

    <hr>

  

  <div role="contentinfo">
    <p>
        &copy; Copyright 2019, Torch Contributors.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              <ul>
<li><a class="reference internal" href="#">Automatic differentiation package - torch.autograd</a><ul>
<li><a class="reference internal" href="#functional-higher-level-api">Functional higher level API</a></li>
<li><a class="reference internal" href="#locally-disabling-gradient-computation">Locally disabling gradient computation</a></li>
<li><a class="reference internal" href="#in-place-operations-on-tensors">In-place operations on Tensors</a><ul>
<li><a class="reference internal" href="#in-place-correctness-checks">In-place correctness checks</a></li>
</ul>
</li>
<li><a class="reference internal" href="#variable-deprecated">Variable (deprecated)</a></li>
<li><a class="reference internal" href="#tensor-autograd-functions">Tensor autograd functions</a></li>
<li><a class="reference internal" href="#function"><span class="hidden-section">Function</span></a></li>
<li><a class="reference internal" href="#context-method-mixins">Context method mixins</a></li>
<li><a class="reference internal" href="#numerical-gradient-checking">Numerical gradient checking</a></li>
<li><a class="reference internal" href="#profiler">Profiler</a></li>
<li><a class="reference internal" href="#anomaly-detection">Anomaly detection</a></li>
</ul>
</li>
</ul>

            </div>
          </div>
        </div>
      </section>
    </div>

  


  

     
       <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
         <script src="_static/jquery.js"></script>
         <script src="_static/underscore.js"></script>
         <script src="_static/doctools.js"></script>
         <script src="_static/language_data.js"></script>
     

  

  <script type="text/javascript" src="_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>
 
<script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-90545585-1', 'auto');
  ga('send', 'pageview');

</script>

<script async src="https://www.googletagmanager.com/gtag/js?id=UA-117752657-2"></script>

<script>
  window.dataLayer = window.dataLayer || [];

  function gtag(){dataLayer.push(arguments);}

  gtag('js', new Date());
  gtag('config', 'UA-117752657-2');
</script>

<img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&amp;guid=ON&amp;script=0"/>


  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/resources">Resources</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.org/assets/brand-guidelines/PyTorch-Brand-Guidelines.pdf" target="_blank">Brand Guidelines</a></li>
          </ul>
        </div>

        <div class="footer-links-col follow-us-col">
          <ul>
            <li class="list-title">Stay Connected</li>
            <li>
              <div id="mc_embed_signup">
                <form
                  action="https://twitter.us14.list-manage.com/subscribe/post?u=75419c71fe0a935e53dfa4a3f&id=91d0dccd39"
                  method="post"
                  id="mc-embedded-subscribe-form"
                  name="mc-embedded-subscribe-form"
                  class="email-subscribe-form validate"
                  target="_blank"
                  novalidate>
                  <div id="mc_embed_signup_scroll" class="email-subscribe-form-fields-wrapper">
                    <div class="mc-field-group">
                      <label for="mce-EMAIL" style="display:none;">Email Address</label>
                      <input type="email" value="" name="EMAIL" class="required email" id="mce-EMAIL" placeholder="Email Address">
                    </div>

                    <div id="mce-responses" class="clear">
                      <div class="response" id="mce-error-response" style="display:none"></div>
                      <div class="response" id="mce-success-response" style="display:none"></div>
                    </div>    <!-- real people should not fill this in and expect good things - do not remove this or risk form bot signups-->

                    <div style="position: absolute; left: -5000px;" aria-hidden="true"><input type="text" name="b_75419c71fe0a935e53dfa4a3f_91d0dccd39" tabindex="-1" value=""></div>

                    <div class="clear">
                      <input type="submit" value="" name="subscribe" id="mc-embedded-subscribe" class="button email-subscribe-button">
                    </div>
                  </div>
                </form>
              </div>

            </li>
          </ul>

          <div class="footer-social-icons">
            <a href="https://www.facebook.com/pytorch" target="_blank" class="facebook"></a>
            <a href="https://twitter.com/pytorch" target="_blank" class="twitter"></a>
            <a href="https://www.youtube.com/pytorch" target="_blank" class="youtube"></a>
          </div>
        </div>
      </div>
    </div>
  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/features">Features</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/hub">PyTorch Hub</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <a href="https://pytorch.org/resources">Resources</a>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Remove any empty p tags that Sphinx adds
      $("[data-tags='null']").remove();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>